# -*- coding: utf-8 -*-
import scrapy
from amazon.items import CatelogItem,ProductItem
from scrapy.conf import settings
from amazon.models import db_connect,create_amazon_table,Catelog,Product
from sqlalchemy.orm import sessionmaker
import requests
import time
import hashlib
import re
import urllib

class ProductSalesSpider(scrapy.Spider):
    name = 'product_sales'
    custom_settings = {
        'ITEM_PIPELINES': {
            'amazon.pipelines.ProductPipeline': 1,
        }
    }
    def __init__(self, page=None, *args, **kwargs):
        super(ProductSalesSpider, self).__init__(*args, **kwargs)
        self.page = page
        self.ticks = int(time.time())
        engine = db_connect()
        self.Session = sessionmaker(bind=engine)
        self.start_urls = self.findUrlsFromDb(self.page)

    def findUrlsFromDb(self,page):
        productUrls = []
        session = self.Session()
        if page is not None:
            _page = int(page)
            pageSize = 1000
            #pageSize = 1
            items = session.query(Product).limit(pageSize).offset(pageSize*(_page-1)).all()
            for product in items:
                productUrls.append(product.product_url)
        return productUrls

    def parse(self, response):
        item = self.handleProductDetail(response)
        yield item # 先保存，再做另外的请求处理销量
        if 'cate_top' in item and 'rank_top' in item:
            salesLink = self.rankTopUrl(item)
            yield scrapy.Request(salesLink,callback=self.parseSalesMonth,meta={'product': item})
        
    def parseSalesMonth(self,response):
        item = response.meta['product']
        item['month_sales'] = response.text
        yield item
    
    def handleProductDetail(self,response):
        item = ProductItem()
        item['product_url'] = response.url
        productRating= response.xpath('//*[@id="acrPopover"]//@title').extract_first()
        if productRating is not None :
            item['product_rating'] = productRating
        productReviews = response.xpath('//*[@id="acrCustomerReviewText"]//text()').extract_first()
        if productReviews is not None :
            item['product_reviews'] = productReviews
        price = response.xpath('//*[@id="priceblock_ourprice"]//text()').extract_first()
        if price is not None:
            item['price_amazon'] = price
            item['product_price'] = price
        ranks = {}
        #if 'bestsellers_rank' in product and product['bestsellers_rank'] is not None:
        #    ranks = item['bestsellers_rank']
        # 主目录排名
        topRankText = ''.join(response.xpath('//*[@id="SalesRank"]/text()').extract())
        topRank = topRankText.split('in')
        if len(topRank) == 2:
            rankNum = int(re.sub(r'\D',"",topRank[0].strip()))
            item['rank_top'] = rankNum
            cateLine = []
            cateTop = topRank[1].replace('()','').strip()
            cateLine.append(cateTop)
            item['cate_top'] = cateTop
            cateId = self.cateId({'cate_line':cateLine})
            rank = {
                'cate_id': cateId,
                'cate_name' : self.cateName({'cate_line':cateLine}),
                'cate_level': len(cateLine),
                'cate_top' : cateLine[0],
                'rank' : rankNum
            }
            ranks[cateId] = rank
        # 子目录排名      
        for ranknode in response.xpath('//*[@id="SalesRank"]/ul/li[@class="zg_hrsr_item"]'):
            cateLine = ranknode.xpath('./span[@class="zg_hrsr_ladder"]//a//text()').extract()
            cateId = self.cateId({'cate_line':cateLine})
            rank = {
                'cate_id': cateId,
                'cate_name' : self.cateName({'cate_line':cateLine}),
                'cate_level': len(cateLine),
                'cate_top' : cateLine[0],
                'rank' : int(re.sub(r'\D',"",ranknode.xpath('./span[@class="zg_hrsr_rank"]//text()').extract_first()))
            }
            ranks[cateId] = rank
        item['bestsellers_rank'] = ranks
        return item
 
    def rankTopUrl(self,product):
        cateTop = urllib.parse.quote(product['cate_top'])
        rankNum = product['rank_top']
        salesLink = "https://www.amztracker.com/unicorn.php?domain=www.amazon.com&category=%s&rank=%s&_=%s" % (cateTop,rankNum,self.ticks)
        return salesLink
    
    def cateId(self,catelog):
        cateName = self.cateName(catelog)
        sha1 = hashlib.sha1()
        sha1.update(cateName.encode('utf-8'))
        return sha1.hexdigest()

    def cateName(self,catelog):
        return " > ".join(catelog['cate_line'])




    